import requests
from parsel import Selector

from proxypool.schemas.proxy import Proxy
from proxypool.crawlers.base import BaseCrawler

requests.packages.urllib3.disable_warnings()
BASE_URL = "https://proxy.seofangfa.com/"
MAX_PAGE = 1


class SeoFangFaCrawler(BaseCrawler):
    """
    seo方法 crawler, https://proxy.seofangfa.com/
    """
    urls = ["https://proxy.seofangfa.com/"]

    def parse(self, html):
        """
        parse html file to get proxies
        :return:
        """
        sel = Selector(html)
        trs = sel.xpath('//table[@class="table"]//tr[position()>1]')
        for tr in trs:
            host = tr.css('td:nth-child(1) *::text').get()
            port = int(tr.css('td:nth-child(2) *::text').extract_first())
            yield Proxy(host=host, port=port)